const axios = require('axios')
const cheerio = require('cheerio')
const crawlab = require('crawlab-sdk')

export const downLoad = async function (url, fn) {
  try {
    const { data: html } = await axios(url);
    const $ = cheerio.load(html)
    fn && fn($)
  } catch (e) {}
}

export default class sipder {
  queue = []

  constructor () {
    this.crawlab = crawlab
  }

  addUrl (url) {
    this.queue.forEach((o) => {
      o.url.match(url) && downLoad(o.url, o.fn)
    })
  }

  addRule (url, fn) {
    this.queue.push({
      url,
      fn
    })
  }
}